import requests
from fake_useragent import UserAgent
from lxml import etree


# 管理，之前爬取过的过滤掉
class URLManager(object):
    def __init__(self):
        self.new_url = []
        self.old_url = []

    # 获取一个url
    def get_new_url(self):
        url = self.new_url.pop()
        self.old_url.append(url)
        return url

    # 增加一个url
    def add_new_url(self, url):
        if url not in self.new_url and url and url not in self.old_url:
            self.old_url.append(url)

    # 增加多个url
    def add_new_urls(self, urls):
        for url in urls:
            self.old_url.append(self.old_url)

    # 判断是否还可以爬去的url
    def has_new_url(self):
        return self.get_new_url_size() > 0

    # 获取可爬取的数量
    def get_new_url_size(self):
        return len(self.new_url)

    # 获取已经爬取的数量
    def get_old_url_size(self):
        return len(self.old_url)


# 爬取
class Dowloader:
    def __init__(self, url):
        self.url = url

    def download(self, url):
        response = requests.get(url, headers={"User-Agent": UserAgent().random})
        if response.status_code == 200:
            response.encoding = 'utf-8'
            return response.text
        else:
            return None


# 解析
class Parser:
    def parse(self, html):
        e = etree.HTML(html)
        data =self.parse_info(e)
        # 下一页
        urls = self.parse_urls(e)
        return data,urls

    def parse_info(self, e):
        spans = e.xpath()
        datas = []
        for span in spans:
            datas.append(span.xpath('string(.)'))
        return datas

    def parse_urls(self,e):
        base_url='...{}'
        urls=[]
        for url in e.xpath(''):
            urls.append(base_url.format(url))
        return urls

# 数据处理
class DataOutPut:
    def save(self, datas):
        with open('少儿编程.txt','a',encoding='utf-8') as f:
            for data in datas:
                f.write(data)

# 调度
class DiaoDu:
    def __init__(self):
        self.downloader = Dowloader()
        self.url_manager = URLManager()
        self.parser = Parser()
        self.data_saver = DataOutPut()

    def run(self, url):
        self.url_manager.add_new_url(url)
        while self.url_manager.has_new_url():
            url = self.url_manager.get_new_url()
            html = self.downloader.download(url)
            data, urls = self.parser.parse(html)
            self.data_saver.save(data)
            self.url_manager.add_new_urls(urls)


if __name__ == '__main__':
    diao_du = DiaoDu()
    diao_du.run("https://www.dongqiudi.com/")
